# -*- coding: utf-8 -*-
import scrapy

from tencent.items import TencentItem


class TencentPositionSpider(scrapy.Spider):
    name = 'tencent'
    allowed_domains = ['tencent.com']
    url = "https://hr.tencent.com/position.php?&start="
    offset = 0
    start_urls = [url + str(offset)]

    def parse(self, response):
        for each in response.xpath('//tr[@class="even"] | //tr[@class="odd"]'):
            # 初始化模型对象
            item = TencentItem()
            item['positionName'] = each.xpath('./td[1]/a/text()').extract()[0]
            item['positionLink'] = each.xpath('./td[1]/a/@href').extract()[0]
            # item['positionType'] = each.xpath('./td[2]/text()').extract()[0] 因为这里可能会报错，数组越界
            item['peopleNum'] = each.xpath('./td[3]/text()').extract()[0]
            item['workLocation'] = each.xpath('./td[4]/text()').extract()[0]
            item['publishTime'] = each.xpath('./td[5]/text()').extract()[0]
            # 将数据交给管道文件处理
            yield item

        # 每次处理完一页的数据之后，重新发送下一页页面请求
        if self.offset < 3950:
            self.offset += 10
        # else:
        #     raise("结束工作")
        # 不加else也可以，因为请求重复了，不会重复发请求
        # 将请求重新发送给调度器入队列，出队列，交给下载器下载
        yield scrapy.Request(self.url + str(self.offset), callback=self.parse)


